#Matan Keidar 066500992
#Eliahu Khalastchi 040854705

V = 300000

import math 

class HeldOut:
    '''
    Represents a Heldout smoothing method
    '''
    
    def __init__(self, corpusT, corpusH):
        '''
        Constructor
        '''
        self.corpusT = corpusT #training corpus
        self.corpusH = corpusH #heldout corpus
        self.trtH = {} 
        self.nrT = {}
        self.updateParams() 
    
    def updateTrTH(self):        
        '''
        Calculates the trTH of each mode (r)
        '''
        
        for mode in set(self.corpusT.getValues()): # for each instance of r in T
           
            sum = 0
            
            words = self.corpusT.filterByMode(mode)     
            for word in words:  # for each word that appeared r times in T
                sum += self.corpusH.C(word) # sum its appearances in H
            
            self.trtH[mode] = sum
            
        sum=0
        wordsH = self.corpusH.getEntries()
        wordsT = self.corpusT.getEntries()
        
        # for each word that appears only in H (not in T)
        for word in wordsH:
            if word not in wordsT:
                sum += self.corpusH.C(word) # sum its appearances in H
                
                    
        self.trtH[0] = sum;
    
    def updateNr(self):
        '''
        Calculates the Nr for each mode (r)
        '''
        
        for mode in set(self.corpusT.getValues()):
            modeCounts = len(self.corpusT.filterByMode(mode))
            self.nrT[mode] = modeCounts
        self.nrT[0] = V - self.corpusT.X()  

    def updateParams(self):
        '''
        Updates the parameters of the heldout model
        '''
        self.updateTrTH()
        self.updateNr() 

    def calcProbability(self, word):
        '''
        Calculates the probability of the given word according to heldout smoothing method
        '''
        
        r = self.corpusT.getWord(word)
        if r is None:
            r = 0
            
        s=self.corpusH.S()        
        p=float(self.trtH[r]) / float((self.nrT[r]*s))
               
        return p
    
    def calcFrequency(self, r):
        '''
        Calculates the r* frequency of the given mode according to Heldout smoothing method
        '''
        return float(self.trtH[r]) / self.nrT[r]
        
    def calcPerplexity(self, testFile):
        '''
        calculates the perplexity of the current model
        '''
        sum = float(0)
        counter = 0
        
        #open test file
        input = open(testFile, "r")
        
        #scan each word in test file and sum its log scaled probability 
        for line in input:
            words = line.strip().split()
            
            for word in words: 
                sum += math.log(self.calcProbability(word))
                counter += 1
                
        input.close()
        
        #return perplexity 
        return math.pow(2, -1.0 / counter * sum)

    def debug(self):
        '''
        Debug current model probabilities
        '''
        sum = float(0)        
        #scan each word in corpus and accumulate its probability
        for word in self.corpusT.getEntries():
            sum += self.calcProbability(word)
            
        
        #accumulate the probabilities of the "unseen" words
        sum += (V - self.corpusT.X()) * self.calcProbability(None); 
        
        return sum
